import requests
from lxml import etree
from pymongo import MongoClient
import time


class Article:
    # 小说头信息对象类
    def __init__(self, img, name, author, type_):
        # 头像地址
        self.img = img
        # 书名
        self.name = name
        # 作者
        self.author = author
        # 类型
        self.type_ = type_


class Volume:
    # 小说章节对象类
    def __init__(self, name, creatime, count):
        # 章节标题
        self.chapterName = name
        # 创建时间
        self.creatime = creatime
        # 内容
        self.count = count

#  全局session
session = requests.session()


def start():
    # 起点免费小说分页url
    url = 'https://www.qidian.com/free/all?orderId=&vip=hidden&style=1&pageSize=20&siteid=1&pubflag=0&hiddenField=1' \
          '&page=%d'
    for i in range(1, 5):
        #  循环获取免费文章
        html_xpath = '/html/body/div/div[@class="free-container box-center"]/div[' \
                     '@id="free-channel-wrap"]/div/div/div[@class="main-content-wrap fl"]/div[' \
                     '@class="all-book-list"]/div/ul/li/div/a/@href'
        # 获取免费小说的url
        html_data = getHtmlData(url % i, html_xpath)
        print(len(html_data))
        for item in html_data:
            # 循环爬取免费小说
            catal_data = getHtmlData('https:' + item + '#Catalog',
                                     '/html/body/div/div[@class="book-detail-wrap center990"]')[0]
            # 获取小说章节目录
            information = catal_data.xpath('./div[@class="book-information cf"]/div')
            # 整理小说头信息
            img = information[0].xpath('./a/img/@src')[0]
            name = information[1].xpath('./h1/em/text()')[0]
            author = information[1].xpath('./h1/span/a/text()')[0]
            type_arr = information[1].xpath('./p[1]/a/text()')
            if len(type_arr) >= 2:
                type_ = type_arr[0] + '·' + type_arr[1]
            else:
                type_ = type_arr[0]
            intro = information[1].xpath('./p[2]/text()')[0]
            article = Article(img, name, author, type_)
            article.__setattr__('intro', intro)
            article.__setattr__('_id', time.time())
            #  连接数据库
            conn = getConnet()
            # 保存小说
            _id = saveDB(conn, article)
            # 如果保存成功
            if id is not None:
                # 获取所有章节的url
                catal_href = catal_data.xpath('./div[@id="j-catalogWrap"]/div[@class="volume-wrap"]/div/ul/li/a/@href')
                for volume_url in catal_href:
                    # 循环获取章节的内容
                    volume_xpath = '/html/body/div[@class="wrap"]/div[@id="j-mainReadContainer"]/div[' \
                                   '@id="j_readMainWrap"]/div[@id="j_chapterBox"]/div/div[contains(' \
                                   '@class,"main-text-wrap")]/div'
                    resp_volume = getHtmlData('https:' + volume_url, volume_xpath)
                    # 整理章节信息
                    chapterName = resp_volume[0].xpath('./h3/span/text()')[0]
                    count = resp_volume[0].xpath('./div[@class="text-info cf"]/div/i[1]/span/text()')[0]
                    creatime = resp_volume[0].xpath('./div[@class="text-info cf"]/div/i[2]/span/text()')[0]
                    volume = Volume(chapterName, creatime, count)
                    volume.__setattr__('content',
                                       etree.tostring(resp_volume[1], encoding='utf8').decode('utf8'))
                    # 保存章节
                    pushVolume(conn, _id, volume)
                    print(volume.chapterName)

# 请求url并通过xpath获取目标数据
def getHtmlData(url, xpath_str):
    resp = session.get(url)
    html = etree.HTML(resp.text)
    return html.xpath(xpath_str)


def getConnet():
    conn = MongoClient('127.0.0.1', 27017)
    return conn.db_article  # 连接mydb数据库，没有则自动创建


def saveDB(db, obj):
    my_set = db.t_free
    # 插入前，先查找是否有相同的书籍
    res = my_set.find_one({'name': obj.name, 'author': obj.author}, {'_id': 1})
    if res is None:
        # 如果没有则插入，并返回刚id
        inset_res = my_set.insert_one(obj.__dict__)
        return inset_res.inserted_id
    # 如果有则直接返回已有的书籍id
    return res['_id']


def pushVolume(db, _id, obj):
    # 章节是list，push前先查找是否
    my_set = db.t_free
    res = my_set.find_one({'_id': _id, 'volume.chapterName': obj.chapterName}, {'_id': 1})
    if res is None:
        my_set.update_many({'_id': _id}, {'$push': {"volume": obj.__dict__}})


if __name__ == '__main__':
    start()
